{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"human_value_alignment_instructions_part2.json\", \"r\", encoding='utf-8') as f:\n",
    "    lines = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'Exception'>\n",
      "<class 'Exception'>\n",
      "<class 'Exception'>\n",
      "<class 'Exception'>\n",
      "<class 'Exception'>\n",
      "<class 'Exception'>\n"
     ]
    }
   ],
   "source": [
    "output = []\n",
    "for line in lines:\n",
    "#     print(line)\n",
    "    try:\n",
    "        if line[-1] == '\\n':\n",
    "            line = line[:-1]\n",
    "        item = json.loads(line)\n",
    "        output.append(item)\n",
    "    except Exception:\n",
    "        print(Exception)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19464"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "json.dump(output, open(\"human_value_alignment_instructions_part2_valid.json\", \"w\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "63532\n"
     ]
    }
   ],
   "source": [
    "outputs1 = []\n",
    "\n",
    "with open(\"exam_instructions.jsonl\", \"r\", encoding=\"utf-8\") as f:\n",
    "    for line in f.readlines():\n",
    "        if line[-1] == '\\n':\n",
    "            line = line[:-1]\n",
    "        item = json.loads(line)\n",
    "        \n",
    "        inputs = []\n",
    "        outputs = []\n",
    "        if item[\"textbox_q_instruction\"]:\n",
    "            assert isinstance(item[\"textbox_q_instruction\"], str), item[\"textbox_q_instruction\"]\n",
    "            inputs.append(item[\"textbox_q_instruction\"])\n",
    "        if item[\"textbox_q_context\"]:\n",
    "            assert isinstance(item[\"textbox_q_context\"], str), item[\"textbox_q_context\"]\n",
    "            inputs.append(item[\"textbox_q_context\"])\n",
    "        if item[\"textbox_question\"]:\n",
    "            assert isinstance(item[\"textbox_question\"], str), item[\"textbox_question\"]\n",
    "            inputs.append(item[\"textbox_question\"])\n",
    "        if item[\"textbox_answer\"]:\n",
    "            assert isinstance(item[\"textbox_answer\"], str)\n",
    "            outputs.append(item[\"textbox_answer\"])\n",
    "        if item[\"textbox_answer_analysis\"]:\n",
    "            assert isinstance(item[\"textbox_answer_analysis\"], str)\n",
    "            outputs.append(item[\"textbox_answer_analysis\"])\n",
    "        \n",
    "        outputs1.append({\n",
    "            \"inputs\": \"\\n\\n\".join(inputs),\n",
    "            \"targets\": \"\\n\\n\".join(outputs),\n",
    "        })\n",
    "\n",
    "print(len(outputs1))\n",
    "json.dump(outputs1, open(\"exam.json\", \"w\"), indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs': '阅读下列短文，从每题所给的四个选项《A、 B、 C和D)中。选出最佳选项。\\n\\nDoctor are known to be terrible pilots. They don\\'t listen because they already know it all. I was lucky: ] became a pilot in 1970, almost ten years before I graduated from medical school. I didn\\'t realize then, but becoming a pilot makes me a better surgeon. I loved flying. As I flew bigger, faster planes, and in worse weather. I learned about crew resource management (机组资源管理), or CRM, a new idea to make flying safer. It means that crew members should listen and speak up for a good result, regardless of positions.    I first read about CRM in 1980. Not long after that, an attending doctor and I were flying in bad weather. The controller had us turn too late to get our landing ready. The attending doctor was flying; I was safety pilot He was so busy because of the bad turn, he had forgotten to put the landing gear (起落架) down. He was a better pilot - and my boss - so it felt unusual to speak up. But I had to: Our lives were in danger. I put aside my uneasiness and said, \"We need to put the landing gear down now!\" That was my first real lesson in the power of CRM, and I\\'ve used it in the operating room ever since. CRM requires that the pilot/surgeon encourage others to speak up. It further requires that when opinions are from the opposite, the doctor doesn\\'t overreact, which might prevent fellow doctors from voicing opinions again. So when I\\'m in the operating room, I ask for ideas and help from others. Sometimes they\\'re not willing to speak up. But I hope that if I continue to encourage them , someday someone will keep me  from ”landing gear up”.\\n\\nWhich of the following can be the best title for the text? A.CRM:A New Way to Make Flying Safe B. Flying Makes Me a Better Doctor C. The Making of a Good Pilot D.A Pilot-Turned Doctor', 'targets': 'B\\n\\n主旨大意题。作者通过当飞行员的经历，学到了CRM，并运用到自己的工作中，文章主要讲的是CRM带给作者的好处，B项“飞行使我成为一个更好的医生”符合题意，故选B。'}\n"
     ]
    }
   ],
   "source": [
    "tmp = json.load(open(\"exam.json\", \"r\", encoding=\"utf-8\"))\n",
    "# tmp = json.load(open(\"exam.json\", \"r\"))\n",
    "print(tmp[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3000\n"
     ]
    }
   ],
   "source": [
    "outputs2 = []\n",
    "with open(\"human_value_alignment_instructions_part1.json\", \"r\", encoding=\"utf-8\") as f:\n",
    "    data = json.load(f)\n",
    "    for item in data:\n",
    "        inputs = []\n",
    "        if item[\"instruction\"]:\n",
    "            inputs.append(item[\"instruction\"])\n",
    "        if item[\"input\"]:\n",
    "            inputs.append(item[\"input\"])\n",
    "            \n",
    "        outputs2.append({\n",
    "            \"inputs\": \"\\n\\n\".join(inputs),\n",
    "            \"targets\": item[\"output\"]\n",
    "        })\n",
    "\n",
    "print(len(outputs2))\n",
    "json.dump(outputs2, open(\"human_align_instructions_part1.json\", \"w\"), indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'inputs': '你在学校里看到同学因为某些原因心情不好，你该怎么做来安慰他们？', 'targets': '当遇到同学处于情绪不好的状况时，可以采用以下措施：首先，你要真正地去理解和感受到对方的情绪，用语言表达出来。然后可以通过耐心地聆听对方的心声和对其的关心和关注来安抚他们的情绪。同时，在适当的时候可以放松气氛，让TA开心起来，可以讲一些有趣的事情，或者播放想听的音乐，做一些适当的活动。不过要尊重TA的情绪，不要贸然说些冷言冷语或者轻易给予建议。通过这些方法，也许可以取得很好的效果，帮助TA渡过难关。'}\n"
     ]
    }
   ],
   "source": [
    "tmp = json.load(open(\"human_align_instructions_part1.json\", \"r\"))\n",
    "print(tmp[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19464\n"
     ]
    }
   ],
   "source": [
    "outputs3 = []\n",
    "with open(\"human_value_alignment_instructions_part2_valid.json\", \"r\", encoding=\"utf-8\") as f:\n",
    "    data = json.load(f)\n",
    "    for item in data:\n",
    "        inputs = []\n",
    "        outputs = []\n",
    "        if item[\"textbox_q_instruction\"]:\n",
    "            assert isinstance(item[\"textbox_q_instruction\"], str), item[\"textbox_q_instruction\"]\n",
    "            inputs.append(item[\"textbox_q_instruction\"])\n",
    "        if item[\"textbox_q_context\"]:\n",
    "            assert isinstance(item[\"textbox_q_context\"], str), item[\"textbox_q_context\"]\n",
    "            inputs.append(item[\"textbox_q_context\"])\n",
    "        if item[\"textbox_question\"]:\n",
    "            assert isinstance(item[\"textbox_question\"], str), item[\"textbox_question\"]\n",
    "            inputs.append(item[\"textbox_question\"])\n",
    "        if item[\"textbox_answer\"]:\n",
    "            assert isinstance(item[\"textbox_answer\"], str)\n",
    "            outputs.append(item[\"textbox_answer\"])\n",
    "        if item[\"textbox_answer_analysis\"]:\n",
    "            assert isinstance(item[\"textbox_answer_analysis\"], str)\n",
    "            outputs.append(item[\"textbox_answer_analysis\"])\n",
    "        \n",
    "        outputs3.append({\n",
    "            \"inputs\": \"\\n\\n\".join(inputs),\n",
    "            \"targets\": \"\\n\\n\".join(outputs),\n",
    "        })\n",
    "\n",
    "print(len(outputs3))\n",
    "json.dump(outputs3, open(\"human_align_insturctions_part2.json\", \"w\"), indent=2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11737\n"
     ]
    }
   ],
   "source": [
    "outputs4 = []\n",
    "with open(\"leetcode_instructions.jsonl\", \"r\", encoding=\"utf-8\") as f:\n",
    "    for line in f.readlines():\n",
    "        if line[-1] == '\\n':\n",
    "            line = line[:-1]\n",
    "        item = json.loads(line)\n",
    "        inputs = []\n",
    "        if item[\"instruction\"]:\n",
    "            inputs.append(item[\"instruction\"])\n",
    "        if item[\"input\"]:\n",
    "            inputs.append(item[\"input\"])\n",
    "        outputs4.append({\n",
    "            \"inputs\": \"\\n\\n\".join(inputs),\n",
    "            \"targets\": item[\"output\"],\n",
    "        })\n",
    "\n",
    "print(len(outputs4))\n",
    "json.dump(outputs4, open(\"leetcode.json\", \"w\"), indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "67798\n"
     ]
    }
   ],
   "source": [
    "outputs5 = []\n",
    "with open(\"translated_instructions.jsonl\", \"r\", encoding=\"utf-8\") as f:\n",
    "    for line in f.readlines():\n",
    "        if line[-1] == \"\\n\":\n",
    "            line = line[:-1]\n",
    "        item = json.loads(line)\n",
    "        inputs = []\n",
    "        if item[\"trans_instruction\"]:\n",
    "            inputs.append(item[\"trans_instruction\"])\n",
    "        if item[\"trans_input\"]:\n",
    "            inputs.append(item[\"trans_input\"])\n",
    "        outputs5.append({\n",
    "            \"inputs\": \"\\n\\n\".join(inputs),\n",
    "            \"targets\": item[\"trans_output\"]\n",
    "        })\n",
    "\n",
    "print(len(outputs5))\n",
    "json.dump(outputs5, open(\"translated_instruct.json\", \"w\"), indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13653\n"
     ]
    }
   ],
   "source": [
    "files = list(glob(\"counterfactural_correction_multi_round_chat/counterfactural_correction_multi_round_chat/*.json\"))\n",
    "print(len(files))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|████████████████████████████████████████████▊                                                                                                                                                                                                                                                                                | 1931/13653 [00:00<00:04, 2724.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Jump counterfactural_correction_multi_round_chat/counterfactural_correction_multi_round_chat\\6NtIe56tsX.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|███████████████████████████████████████████████████████████████████████████▍                                                                                                                                                                                                                                                 | 3251/13653 [00:01<00:04, 2567.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Jump counterfactural_correction_multi_round_chat/counterfactural_correction_multi_round_chat\\bKj7AdyEZK.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                                                                                                                                                                                                 | 5333/13653 [00:02<00:03, 2561.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Jump counterfactural_correction_multi_round_chat/counterfactural_correction_multi_round_chat\\fygBVQf00R.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████         | 13265/13653 [00:05<00:00, 2559.42it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Jump counterfactural_correction_multi_round_chat/counterfactural_correction_multi_round_chat\\Y6racS6pSM.json\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 13653/13653 [00:05<00:00, 2525.87it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "13649\n"
     ]
    }
   ],
   "source": [
    "outputs6 = []\n",
    "for file in tqdm(files, total=len(files)):\n",
    "    try:\n",
    "        data = json.load(open(file, 'r', encoding='utf-8'))\n",
    "    except:\n",
    "        print(f\"Jump {file}\")\n",
    "        continue\n",
    "    keys = sorted([int(k[6:]) for k in data.keys() if k.startswith(\"round_\")])\n",
    "    dialog = []\n",
    "    for k in keys:\n",
    "        turn = data[f\"round_{k}\"]\n",
    "        dialog.append({\n",
    "            \"inputs\": turn[\"prompt\"],\n",
    "            \"targets\": turn[\"response\"],\n",
    "        })\n",
    "    outputs6.append(dialog)\n",
    "\n",
    "print(len(outputs6))\n",
    "json.dump(outputs6, open(\"dialogs.json\", \"w\"), indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dialogs.json\n",
      "13649\n",
      "exam.json\n",
      "63532\n",
      "human_align_instructions_part1.json\n",
      "3000\n",
      "human_align_insturctions_part2.json\n",
      "19464\n",
      "leetcode.json\n",
      "11737\n",
      "translated_instruct.json\n",
      "67798\n",
      "179180\n"
     ]
    }
   ],
   "source": [
    "all_data = []\n",
    "for file in [\"dialogs.json\", \"exam.json\", \"human_align_instructions_part1.json\", \"human_align_insturctions_part2.json\", \"leetcode.json\", \"translated_instruct.json\"]:\n",
    "    print(file)\n",
    "    sub_data = json.load(open(file, \"r\", encoding=\"utf-8\"))\n",
    "    print(len(sub_data))\n",
    "    all_data.extend(sub_data)\n",
    "print(len(all_data))\n",
    "json.dump(all_data, open(\"coig_data.json\", \"w\"), indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
